package othello.players.learnplayers;

import java.util.ArrayList;
import java.util.List;
import java.util.ListIterator;
import java.util.Random;
import java.awt.Color;

import othello.model.Action;
import othello.model.IBoard;
import othello.model.State;
import othello.neuralnetwork.NeuralNetwork;

public class HeurTD extends AbstractNeuralNetworkPlayer {
    private static final double epsInit = 0.1;
    private static final double tBase = 0.9999995;    // got the t-values from a paper
    private static final double tThreshold = 0.002;	 
    private static final double lambda_decay = 0.75;
    private final boolean softmax = false;
    private final boolean epsgreedy = true;
    private List<State> previousStates = new ArrayList<State>();    
    private State lastState;
    private int totalGames = -1;
    
    private static final int[][] positionValues = 
		new int[][]{{100,-25, 10,  5,  5, 10,-25,100},
                     {-25,-25,  2,  2,  2,  2,-25,-25},
                     { 10,  2,  5,  1,  1,  5,  2, 10},
                     {  5,  2,  1,  2,  2,  1,  2,  5},
                     {  5,  2,  1,  2,  2,  1,  2,  5},
                     { 10,  2,  5,  1,  1,  5,  2, 10},
                     {-25,-25,  2,  2,  2,  2,-25,-25},
                     {100,-25, 10,  5,  5, 10,-25,100}};
    
    public HeurTD(NeuralNetwork neuralNetwork) {
        super("HeurTD", neuralNetwork);
    }
    
    @Override
    public void beginTurn() {
    	if (totalGames == -1){
    		totalGames = board.getRealMaxGames();
    	}
    	
    	//select which action to take according to some policy
    	
    	//Action action = selectActionPos;
        Action action = selectActionPos();
        IBoard hypoBoard = board;
        
        if (action != null)        
        	hypoBoard = board.hypotheticallyPlacePiece(this.color, action.getRow(), action.getCol());
                
    	State state = new State(cloneByteArray(hypoBoard.getState()));
        
        // update NN        
        if (this.benchmarkState == BenchmarkState.Training && lastState != null){        	
        	nn.forwardPropagate(state.toDoubles());
        	double newV = nn.getOutput()[0];
        	nn.forwardPropagate(lastState.toDoubles());
        	double oldV = nn.getOutput()[0];
        	double error = newV - oldV;
        	
        	if(useEligibility){
        		previousStates.add(state);
        		double lambda = 1;
                
                ListIterator<State> li = previousStates.listIterator(previousStates.size());   
                State stateItr = li.previous();
                
                while(li.hasPrevious()) {
                	stateItr = li.previous();
                    nn.forwardPropagate(stateItr.toDoubles());
                    double decayedError = lambda*error;
                    nn.backPropagate(decayedError, nnLR, 0);
                    lambda *= lambda_decay;    	        	
                }        		
        		
        	} else {
        		nn.backPropagate(error, nnLR, 0);
        	}
        }               

        if (action == null){
        	board.pass(this);
        } else {            
            lastState = state;
        	board.placePiece(this, action.getRow(), action.getCol());
        }
    }
    
	@Override
    public void endGame() {
		// Update NN
    	if (this.benchmarkState == BenchmarkState.Training && lastState != null) {
    		nn.forwardPropagate(lastState.toDoubles());
        	double oldV = nn.getOutput()[0];
        	double reward = reward(board);        	
        	double error = reward - oldV;
        	//System.out.print("Reward " + getColor() + ": " + reward + ", oldV: " + oldV + '\n');
        	
        	
        	if(useEligibility){
        		double lambda = 1;                
                ListIterator<State> li = previousStates.listIterator(previousStates.size());                                
                while(li.hasPrevious()) {
                    State stateItr = li.previous();
                    nn.forwardPropagate(stateItr.toDoubles());
                    double decayedError = lambda*error;
                    nn.backPropagate(decayedError, nnLR, 0);
                    lambda *= lambda_decay;    	        	
                }        		
        		
        	} else {
        		nn.backPropagate(error, nnLR, 0);
        	}
        	
        	lastState = null;
        	this.previousStates.clear();
        	
    	}
    }       

	
	private Action selectActionPos()
	{	
		int maxEval = Integer.MIN_VALUE;
		int bestR = -1;
		int bestC = -1;
		int eval;
		
		for (int r = 0; r < board.getRowCount(); r++) {
			for (int c = 0; c < board.getColCount(); c++) {
				if (board.isValidMove(getColor(), r, c)) {    
					IBoard hypoBrd = board.hypotheticallyPlacePiece(this.getColor(), r, c);						
					eval = evalWeighted(hypoBrd);						
					if (eval > maxEval){
						maxEval = eval;
						bestR = r;
						bestC = c;
					}
				}
			}         
		}
		
		if (bestR == -1){
			return null;
		}
		
		return new Action(bestR,bestC);	
	}
		
	private int evalWeighted(IBoard brd) {
        int eval = 0;
        for (int r = 0; r < brd.getRowCount(); r++) {
            for (int c = 0; c < brd.getColCount(); c++) {
                eval += positionValues[r][c] * positionOccupance(brd,r,c);
            }
        }
        return eval;
    }
    
    private int positionOccupance(IBoard brd, int r, int c) {
        Color color = brd.playerAt(r,c);
        if (color == null){
            return 0;
        }
        if (color.equals(this.getColor())) {
            return 1;
        }
        
        return -1;
    }    
    
    @Override
    public void endTurn() {
    }
}
